STAT 6021: Project 2

Load in packages

library(tidyverse)

Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
── Attaching packages ──────────────────────────────────────────────────────────────────────────────────── tidyverse 1.3.1 ──
✓ ggplot2 3.3.5     ✓ purrr   0.3.4
✓ tibble  3.1.4     ✓ dplyr   1.0.7
✓ tidyr   1.1.3     ✓ stringr 1.4.0
✓ readr   2.0.1     ✓ forcats 0.5.1
── Conflicts ─────────────────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()

library(tidyverse)
library(ggplot2)

import data:

find missing data:

No missing data. Check data types of each variable:

str(house)

'data.frame':   21613 obs. of  22 variables:
 $ id           : num  7.13e+09 6.41e+09 5.63e+09 2.49e+09 1.95e+09 ...
 $ date         : chr  "20141013T000000" "20141209T000000" "20150225T000000" "20141209T000000" ...
 $ price        : num  221900 538000 180000 604000 510000 ...
 $ bedrooms     : int  3 3 2 4 3 4 3 3 3 3 ...
 $ bathrooms    : num  1 2.25 1 3 2 4.5 2.25 1.5 1 2.5 ...
 $ sqft_living  : int  1180 2570 770 1960 1680 5420 1715 1060 1780 1890 ...
 $ sqft_lot     : int  5650 7242 10000 5000 8080 101930 6819 9711 7470 6560 ...
 $ floors       : num  1 2 1 1 1 1 2 1 1 2 ...
 $ waterfront   : int  0 0 0 0 0 0 0 0 0 0 ...
 $ view         : int  0 0 0 0 0 0 0 0 0 0 ...
 $ condition    : int  3 3 3 5 3 3 3 3 3 3 ...
 $ grade        : int  7 7 6 7 8 11 7 7 7 7 ...
 $ sqft_above   : int  1180 2170 770 1050 1680 3890 1715 1060 1050 1890 ...
 $ sqft_basement: int  0 400 0 910 0 1530 0 0 730 0 ...
 $ yr_built     : int  1955 1951 1933 1965 1987 2001 1995 1963 1960 2003 ...
 $ yr_renovated : int  0 1991 0 0 0 0 0 0 0 0 ...
 $ zipcode      : int  98178 98125 98028 98136 98074 98053 98003 98198 98146 98038 ...
 $ lat          : num  47.5 47.7 47.7 47.5 47.6 ...
 $ long         : num  -122 -122 -122 -122 -122 ...
 $ sqft_living15: int  1340 1690 2720 1360 1800 4760 2238 1650 1780 2390 ...
 $ sqft_lot15   : int  5650 7639 8062 5000 7503 101930 6819 9711 8113 7570 ...
 $ num          : int  1 1 1 1 1 1 1 1 1 1 ...

We will definitely need to change the data type for the date column, and potentially look into creating factors for some of the more ordinal variables.

Convert date variabe to date type:

Turning view, condition, and grade into ordered factors:

Part 2: EDA

library(Hmisc)

Loading required package: lattice
Loading required package: survival
Loading required package: Formula
Registered S3 methods overwritten by 'htmltools':
  method               from         
  print.html           tools:rstudio
  print.shiny.tag      tools:rstudio
  print.shiny.tag.list tools:rstudio
Registered S3 method overwritten by 'htmlwidgets':
  method           from         
  print.htmlwidget tools:rstudio

Attaching package: ‘Hmisc’

The following objects are masked from ‘package:dplyr’:

    src, summarize

The following objects are masked from ‘package:base’:

    format.pval, units

library(Hmisc)
library(tidyverse)
library(dplyr)
library(faraway)

Attaching package: ‘faraway’

The following objects are masked from ‘package:survival’:

    rats, solder

The following object is masked from ‘package:lattice’:

    melanoma

library(Hmisc)
library(tidyverse)
library(dplyr)
library(faraway)
library(gridExtra)

Attaching package: ‘gridExtra’

The following object is masked from ‘package:dplyr’:

    combine

names(house)

 [1] "id"            "date"          "price"         "bedrooms"      "bathrooms"     "sqft_living"   "sqft_lot"     
 [8] "floors"        "waterfront"    "view"          "condition"     "grade"         "sqft_above"    "sqft_basement"
[15] "yr_built"      "yr_renovated"  "zipcode"       "lat"           "long"          "sqft_living15" "sqft_lot15"   
[22] "num"          

Prior to dropping Date and Geotags consider using them for plotting, for example transaction counts by dates?

names(house)

 [1] "price"         "bedrooms"      "bathrooms"     "sqft_living"   "sqft_lot"      "floors"        "waterfront"   
 [8] "view"          "condition"     "grade"         "sqft_above"    "sqft_basement" "yr_built"      "yr_renovated" 
[15] "sqft_living15" "sqft_lot15"   

describe(house)

house 

 16  Variables      21613  Observations
-----------------------------------------------------------------------------------------------------------------------------
price 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0     3625        1   540182   329562   210000   245000   321950   450000   645000   887000  1160000 

lowest :   75000   78000   80000   81000   82000, highest: 5350000 5570000 6890000 7060000 7700000
-----------------------------------------------------------------------------------------------------------------------------
bedrooms 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0       13    0.871    3.371    0.946        2        2        3        3        4        4        5 

lowest :  0  1  2  3  4, highest:  8  9 10 11 33
                                                                                        
Value          0     1     2     3     4     5     6     7     8     9    10    11    33
Frequency     13   199  2760  9824  6882  1601   272    38    13     6     3     1     1
Proportion 0.001 0.009 0.128 0.455 0.318 0.074 0.013 0.002 0.001 0.000 0.000 0.000 0.000
-----------------------------------------------------------------------------------------------------------------------------
bathrooms 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0       30    0.974    2.115   0.8444     1.00     1.00     1.75     2.25     2.50     3.00     3.50 

lowest : 0.00 0.50 0.75 1.00 1.25, highest: 6.50 6.75 7.50 7.75 8.00
-----------------------------------------------------------------------------------------------------------------------------
sqft_living 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0     1038        1     2080    978.4      940     1090     1427     1910     2550     3250     3760 

lowest :   290   370   380   384   390, highest:  9640  9890 10040 12050 13540
-----------------------------------------------------------------------------------------------------------------------------
sqft_lot 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0     9782        1    15107    17855     1800     3322     5040     7618    10688    21398    43339 

lowest :     520     572     600     609     635, highest:  982998 1024068 1074218 1164794 1651359
-----------------------------------------------------------------------------------------------------------------------------
floors 
       n  missing distinct     Info     Mean      Gmd 
   21613        0        6    0.823    1.494   0.5563 

lowest : 1.0 1.5 2.0 2.5 3.0, highest: 1.5 2.0 2.5 3.0 3.5
                                              
Value        1.0   1.5   2.0   2.5   3.0   3.5
Frequency  10680  1910  8241   161   613     8
Proportion 0.494 0.088 0.381 0.007 0.028 0.000
-----------------------------------------------------------------------------------------------------------------------------
waterfront 
       n  missing distinct 
   21613        0        2 
                      
Value          0     1
Frequency  21450   163
Proportion 0.992 0.008
-----------------------------------------------------------------------------------------------------------------------------
view 
       n  missing distinct 
   21613        0        5 

lowest : 0 1 2 3 4, highest: 0 1 2 3 4
                                        
Value          0     1     2     3     4
Frequency  19489   332   963   510   319
Proportion 0.902 0.015 0.045 0.024 0.015
-----------------------------------------------------------------------------------------------------------------------------
condition 
       n  missing distinct 
   21613        0        5 

lowest : 1 2 3 4 5, highest: 1 2 3 4 5
                                        
Value          1     2     3     4     5
Frequency     30   172 14031  5679  1701
Proportion 0.001 0.008 0.649 0.263 0.079
-----------------------------------------------------------------------------------------------------------------------------
grade 
       n  missing distinct 
   21613        0       12 

lowest : 1  3  4  5  6 , highest: 9  10 11 12 13
                                                                                  
Value          1     3     4     5     6     7     8     9    10    11    12    13
Frequency      1     3    29   242  2038  8981  6068  2615  1134   399    90    13
Proportion 0.000 0.000 0.001 0.011 0.094 0.416 0.281 0.121 0.052 0.018 0.004 0.001
-----------------------------------------------------------------------------------------------------------------------------
sqft_above 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0      946        1     1788    876.2      850      970     1190     1560     2210     2950     3400 

lowest :  290  370  380  384  390, highest: 7880 8020 8570 8860 9410
-----------------------------------------------------------------------------------------------------------------------------
sqft_basement 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0      306    0.776    291.5    422.2        0        0        0        0      560      970     1190 

lowest :    0   10   20   40   50, highest: 3260 3480 3500 4130 4820
-----------------------------------------------------------------------------------------------------------------------------
yr_built 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0      116        1     1971    33.38     1915     1926     1951     1975     1997     2007     2011 

lowest : 1900 1901 1902 1903 1904, highest: 2011 2012 2013 2014 2015
-----------------------------------------------------------------------------------------------------------------------------
yr_renovated 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0       70    0.122     84.4    161.7        0        0        0        0        0        0        0 

lowest :    0 1934 1940 1944 1945, highest: 2011 2012 2013 2014 2015
                                                                                                                      
Value          0  1935  1940  1945  1950  1955  1960  1965  1970  1975  1980  1985  1990  1995  2000  2005  2010  2015
Frequency  20699     1     2     6     4    13    12    16    27    25    43    88    99    84   112   156    82   144
Proportion 0.958 0.000 0.000 0.000 0.000 0.001 0.001 0.001 0.001 0.001 0.002 0.004 0.005 0.004 0.005 0.007 0.004 0.007

For the frequency table, variable is rounded to the nearest 5
-----------------------------------------------------------------------------------------------------------------------------
sqft_living15 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0      777        1     1987    743.2     1140     1256     1490     1840     2360     2930     3300 

lowest :  399  460  620  670  690, highest: 5600 5610 5790 6110 6210
-----------------------------------------------------------------------------------------------------------------------------
sqft_lot15 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25      .50      .75      .90      .95 
   21613        0     8689        1    12768    13404     1999     3667     5100     7620    10083    17852    37063 

lowest :    651    659    660    748    750, highest: 434728 438213 560617 858132 871200
-----------------------------------------------------------------------------------------------------------------------------

ggplot(house, aes(x=sqft_living, y=price, color=waterfront))+
  geom_point()+
  geom_smooth(method = "lm", se=FALSE)+
  labs(x="sqft_living", 
       y="price",
       title="Scatter plot of price against sqft_living with waterfornt indicator")

`geom_smooth()` using formula 'y ~ x'

ggplot(house, aes(x=sqft_living, y=price, color=view))+
  geom_point()+
  geom_smooth(method = "lm", se=FALSE)+
  labs(x="sqft_living", 
       y="price",
       title="Scatter plot of price against sqft_living with view indicator")

`geom_smooth()` using formula 'y ~ x'

ggplot(house, aes(x=sqft_living, y=price, color=condition))+
  geom_point()+
  geom_smooth(method = "lm", se=FALSE)+
  labs(x="sqft_living", 
       y="price",
       title="Scatter plot of price against sqft_living with condition indicator")

`geom_smooth()` using formula 'y ~ x'

ggplot(house, aes(x=sqft_living, y=price, color=grade))+
  geom_point()+
  geom_smooth(method = "lm", se=FALSE)+
  labs(x="sqft_living", 
       y="price",
       title="Scatter plot of price against sqft_living with grade indicator")

`geom_smooth()` using formula 'y ~ x'

Question: how to deal with indicator (ordinary) varibales in this case?

# Changing `view` to 0 for regular view and 1 for every other view
house$view <- asfactor(ifelse(house$view!=0, 1, 0))

Error in asfactor(ifelse(house$view != 0, 1, 0)) : 
  could not find function "asfactor"

Checking possible interactions after mapping categorical variables to a larger classes

ggplot(house, aes(x=sqft_living, y=price, color=view))+
  geom_point()+
  geom_smooth(method = "lm", se=FALSE)+
  labs(x="sqft_living", 
       y="price",
       title="Scatter plot of price against sqft_living with view indicator")

`geom_smooth()` using formula 'y ~ x'

ggplot(house, aes(x=sqft_living, y=price, color=condition))+
  geom_point()+
  geom_smooth(method = "lm", se=FALSE)+
  labs(x="sqft_living", 
       y="price",
       title="Scatter plot of price against sqft_living with condition indicator")

`geom_smooth()` using formula 'y ~ x'

ggplot(house, aes(x=sqft_living, y=price, color=grade))+
  geom_point()+
  geom_smooth(method = "lm", se=FALSE)+
  labs(x="sqft_living", 
       y="price",
       title="Scatter plot of price against sqft_living with grade indicator")

`geom_smooth()` using formula 'y ~ x'

Converting quantitative predictor floors to a factor 1, 2, 3.

house$floors <- ifelse(house$floors < 2, 1, iflese(house$floors < 3, 2, eflse(>=3, 3, 0)))

Error: unexpected '>=' in "house$floors <- ifelse(house$floors < 2, 1, iflese(house$floors < 3, 2, eflse(>="

ggplot(house, aes(x=sqft_living, y=price, color=floors))+
  geom_point()+
  geom_smooth(method = "lm", se=FALSE)+
  labs(x="sqft_living", 
       y="price",
       title="Scatter plot of price against sqft_living with floors indicator")

`geom_smooth()` using formula 'y ~ x'

Computing age of the house

unique((house$bathrooms))

 [1] 1.00 2.25 3.00 2.00 4.50 1.50 2.50 1.75 2.75 3.25 4.00 3.50 0.75 4.75 5.00 4.25 3.75 0.00 1.25 5.25 6.00 0.50 5.50 6.75
[25] 5.75 8.00 7.50 7.75 6.25 6.50

names(house)

 [1] "price"         "bedrooms"      "bathrooms"     "sqft_living"   "sqft_lot"      "floors"        "waterfront"   
 [8] "view"          "condition"     "grade"         "sqft_above"    "sqft_basement" "sqft_living15" "sqft_lot15"   
[15] "age"          

Other stuff

dp1<-ggplot(house, aes(x=sqft_living, color=waterfront))+
geom_density()+
labs(title="Sqft_living by waterfront")


dp2<-ggplot(house,aes(x=sqft_lot, color=waterfront))+
geom_density()+
labs(title="Sqft_lot by waterfront")


dp3<-ggplot(house,aes(x=sqft_above, color=waterfront))+
geom_density()+
labs(title="Sqft_above by waterfront")


dp4<-ggplot(house,aes(x=sqft_basement, color=waterfront))+
geom_density()+
labs(title="Sqft_basement by waterfront")

dp5<-ggplot(house,aes(x=sqft_living15, color=waterfront))+
geom_density()+
labs(title="Sqft_living15 by waterfront")


dp6<-ggplot(house,aes(x=sqft_lot15, color=waterfront))+
geom_density()+
labs(title="Sqft_lot15 by waterfront")


##produce the 4 density plots in a 2 by 2 matrix
grid.arrange(dp1, dp2, dp3, dp4, dp5, dp6, ncol = 2, nrow = 3)

dp1<-ggplot(house, aes(x=yr_built, color=waterfront))+
geom_density()+
labs(title="Yr_built by waterfront")


dp2<-ggplot(house,aes(x=yr_renovated, color=waterfront))+
geom_density()+
labs(title="Yr_renovated by waterfront")


dp3<-ggplot(house,aes(x=floors, color=waterfront))+
geom_density()+
labs(title="Floors by waterfront")


dp4<-ggplot(house,aes(x=bedrooms, color=waterfront))+
geom_density()+
labs(title="Bedrooms by waterfront")

dp5<-ggplot(house,aes(x=bathrooms, color=waterfront))+
geom_density()+
labs(title="Bathrooms by waterfront")


dp6<-ggplot(house,aes(x=grade, color=waterfront))+
geom_density()+
labs(title="Grade by waterfront")


##produce the 4 density plots in a 2 by 2 matrix
grid.arrange(dp1, dp2, dp3, dp4, dp5, dp6, ncol = 2, nrow = 3)

corr <- round(cor(house[,c("price",quant_vars)]), 1)

Error in `[.data.frame`(house, , c("price", quant_vars)) : 
  undefined columns selected

TODO: interpreting models https://cran.r-project.org/web/packages/jtools/vignettes/summ.html

summary(fit)

Call:
lm(formula = price ~ . - sqft_basement, data = house)

Residuals:
     Min       1Q   Median       3Q      Max 
-1454470  -122952   -15571    98278  4289752 

Coefficients:
                Estimate Std. Error t value Pr(>|t|)    
(Intercept)    1.530e+05  1.429e+04  10.704  < 2e-16 ***
bedrooms      -5.136e+04  2.177e+03 -23.590  < 2e-16 ***
bathrooms      6.255e+04  3.752e+03  16.671  < 2e-16 ***
sqft_living    2.249e+02  4.890e+00  45.982  < 2e-16 ***
sqft_lot      -1.782e-02  5.529e-02  -0.322    0.747    
floors2       -2.660e+03  4.945e+03  -0.538    0.591    
floors3        1.655e+05  1.063e+04  15.579  < 2e-16 ***
waterfront.L   4.684e+05  1.347e+04  34.782  < 2e-16 ***
view1          1.224e+05  5.941e+03  20.605  < 2e-16 ***
condition1    -1.577e+04  3.739e+03  -4.217 2.48e-05 ***
grade1        -3.539e+04  3.646e+03  -9.706  < 2e-16 ***
sqft_above     2.576e+01  4.901e+00   5.256 1.49e-07 ***
sqft_living15  6.649e+01  3.757e+00  17.698  < 2e-16 ***
sqft_lot15    -7.498e-01  8.443e-02  -8.881  < 2e-16 ***
age            2.691e+03  7.745e+01  34.745  < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 233000 on 21598 degrees of freedom
Multiple R-squared:  0.5981,    Adjusted R-squared:  0.5979 
F-statistic:  2296 on 14 and 21598 DF,  p-value: < 2.2e-16

summary(reduced)

Call:
lm(formula = log(price) ~ sqft_living + bedrooms + bathrooms + 
    floors + waterfront + age + sqft_living15 + sqft_lot15, data = house)

Residuals:
    Min      1Q  Median      3Q     Max 
-2.7607 -0.2460  0.0127  0.2358  2.0011 

Coefficients:
                Estimate Std. Error t value Pr(>|t|)    
(Intercept)    1.201e+01  1.980e-02 606.583  < 2e-16 ***
sqft_living    2.772e-04  5.309e-06  52.213  < 2e-16 ***
bedrooms      -4.947e-02  3.223e-03 -15.353  < 2e-16 ***
bathrooms      1.206e-01  5.521e-03  21.843  < 2e-16 ***
floors2        6.553e-02  6.471e-03  10.127  < 2e-16 ***
floors3        3.838e-01  1.542e-02  24.883  < 2e-16 ***
waterfront.L   3.844e-01  1.955e-02  19.667  < 2e-16 ***
age            4.611e-03  1.097e-04  42.017  < 2e-16 ***
sqft_living15  1.965e-04  5.385e-06  36.492  < 2e-16 ***
sqft_lot15    -6.949e-07  8.941e-08  -7.772 8.08e-15 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.348 on 21603 degrees of freedom
Multiple R-squared:  0.5638,    Adjusted R-squared:  0.5636 
F-statistic:  3103 on 9 and 21603 DF,  p-value: < 2.2e-16

tiny <- lm(price ~ sqft_living15+bathrooms+waterfront, 
              data = house)
summary(tiny)

plot(tiny)

plot(house$price, house$sqft_living)

plot(house$price, house$bathrooms)

install.packages("plotly")

library(plotly)
fig = list(
  data = list(
    list(
      x = house$floors,
      y = house$bedrooms,
      type = 'bar'
    )
  ),
  layout = list(
    title = 'A Figure Specified By R List',
    plot_bgcolor='#e5ecf6', 
         xaxis = list( 
           zerolinecolor = '#ffff', 
           zerolinewidth = 2, 
           gridcolor = 'ffff'), 
         yaxis = list( 
           zerolinecolor = '#ffff', 
           zerolinewidth = 2, 
           gridcolor = 'ffff')
  )
)
# To display the figure defined by this list, use the plotly_build function
plotly_build(fig)